package com.flying.test;

import io.vavr.collection.List;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.UUID;

public class WebCrawler {
    public static void main(String[] args) {
        List<String> urlsToCrawl = List.of("https://masterdevskills.com");
        WebCrawler webCrawler = new WebCrawler();
        webCrawler.crawl(urlsToCrawl);
    }

    public void crawl(List<String> urlsToCrawl) {
        urlsToCrawl.toStream()
                .map(ThrowingFunction.unchecked(URL::new))
                .forEach(ThrowingConsumer.unchecked(this::save));
    }

    private void consume(URL url) {
        System.out.println(url.toString());
    }

    private void save(URL url) throws IOException {
        String uuid = UUID.randomUUID().toString();
        InputStream inputStream = url.openConnection().getInputStream();
        Files.copy(inputStream, Paths.get(uuid + ".txt"), StandardCopyOption.REPLACE_EXISTING);
    }
}
